from pymongo import MongoClient
from pprint import pprint

import difflib
import random
import os

client = MongoClient('47.104.130.19', 27017)
db = client['knx_posts_db']
collection = db['offical_posts_coll']

# qiancheng_db = client['qiancheng_db']
# qiancheng_posts, qiancheng_corps = qiancheng_db['qiancheng_posts'], qiancheng_db['qiancheng_corps']

# cursor = collection.aggregate([{'$sample': {'size': 10}}])

# items = []
#
# for item in cursor:
#     items.append(item)
#
# random.shuffle(items)

# cursor = qiancheng_posts.find({
#     'subIndustry': '银行'
# }).limit(50)

# passed = []
# deleted = []
#
# files = [(file, open('offical_train/' + file).read()) for file in os.listdir('offical_train/') if file != '.DS_Store']
#
# for i in range(len(files)):
#     for j in range(len(files)):
#         if i == j:
#             continue
#
#         if len(files[i][1].split()) > 30 or len(files[j][1].split()) > 30:
#             continue
#
#         if (files[i][0], files[j][0]) in passed or (files[j][0], files[i][0]) in passed:
#             continue
#
#         score = difflib.SequenceMatcher(None, files[i][1], files[j][1]).quick_ratio()
#
#         if score > 0.8:
#             print('=' * 130)
#             print(files[i])
#             print('-' * 130)
#             print(files[j])
#
#             ans = input('\n准备删除其中一个吗?(y/n):')
#             if ans == 'y':
#                 deleted.append((files[i][0]))
#                 # if os.path.exists('offical_train/' + files[i][0]):
#                 #     os.remove('offical_train/' + files[i][0])
#                 #     print('delete' + files[i][0])
#                 #     clear()
#
#         passed.append((files[i][0], files[j][0]))
#
# print(deleted)
#

files = [open('offical_train/' + file).read() for file in os.listdir('offical_train/') if file != '.DS_Store']

for j in range(300):
    for item in collection.aggregate([{'$sample': {'size': 1}}]):
        repeated = False

        # 判断是否在内容上有重复
        for file in files:
            score = difflib.SequenceMatcher(None, file, item['description']).quick_ratio()
            if score > 0.8:
                repeated = True
                break

        if repeated:
            continue

        print('-' * 60 + '[' + str(len(files)) + ']' + '-' * 60)
        print(item['name'])
        pprint(item['description'])
        ans = input('\n请输入你所认为的职位类型:')

        if len(ans) == 0:
            continue

        if ans == 'q':
            break

        try:
            f = open('offical_train/' + ans + '-' + str(item['_id']), 'w')
            f.write(item['description'])
            f.close()
            files.append(item['description'])
        except:
            continue
